{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e382de80",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container {width:100% !important; }</style>\"))\n",
    "\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import hvplot.pandas\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "import datetime as dt\n",
    "%autosave 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff1f087e",
   "metadata": {},
   "outputs": [],
   "source": [
    "CPU_FREQ=2.60\n",
    "\n",
    "rdtsc_df_dict = []\n",
    "ttt_df_dict = []\n",
    "for filename in glob.glob(\"../exchange*.log\") + glob.glob(\"../*_1.log\"):\n",
    "    print('processing {}'.format(filename))\n",
    "    for line in open(filename):\n",
    "        tokens = line.strip().split(' ')\n",
    "        if len(tokens) != 4:\n",
    "            continue\n",
    "\n",
    "        try:\n",
    "            time = tokens[0]\n",
    "            tag = tokens[2]\n",
    "            latency = float(tokens[3])\n",
    "            latency_rdtsc = latency / CPU_FREQ\n",
    "            time_datetime = pd.to_datetime(time, format='%H:%M:%S.%f')\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "        if ' RDTSC ' in line:\n",
    "            if tokens[1] != 'RDTSC':\n",
    "                continue\n",
    "\n",
    "            rdtsc_df_dict.append({'timestamp':time, 'tag':tag, 'latency':latency_rdtsc})\n",
    "        elif ' TTT ' in line:\n",
    "            if tokens[1] != 'TTT':\n",
    "                continue\n",
    "\n",
    "            ttt_df_dict.append({'timestamp':time, 'tag':tag, 'latency':latency})\n",
    "        \n",
    "rdtsc_df = pd.DataFrame.from_dict(rdtsc_df_dict)\n",
    "rdtsc_df = rdtsc_df.drop_duplicates().sort_values(by='timestamp')\n",
    "rdtsc_df['timestamp'] = pd.to_datetime(rdtsc_df['timestamp'], format='%H:%M:%S.%f')\n",
    "\n",
    "ttt_df = pd.DataFrame.from_dict(ttt_df_dict)\n",
    "ttt_df = ttt_df.drop_duplicates().sort_values(by='timestamp')\n",
    "ttt_df['timestamp'] = pd.to_datetime(ttt_df['timestamp'], format='%H:%M:%S.%f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55f30ebe",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for tag in rdtsc_df['tag'].unique():\n",
    "    print(tag)\n",
    "    \n",
    "    fig = go.Figure()\n",
    "\n",
    "    t_df = rdtsc_df[rdtsc_df['tag'] == tag].copy()\n",
    "    t_df = t_df[t_df['latency'] > 0]\n",
    "\n",
    "    q_hi = t_df['latency'].quantile(0.99)\n",
    "    q_lo = t_df['latency'].quantile(0.01)\n",
    "    t_df = t_df[(t_df['latency'] < q_hi) & (t_df['latency'] > q_lo)]\n",
    "\n",
    "    mean = t_df['latency'].astype(float).mean()\n",
    "    print('{} has {} observations mean {}'.format(tag, len(t_df), mean))\n",
    "\n",
    "    rolling_window = max(1, int(len(t_df) / 100))\n",
    "\n",
    "    use_micros = False\n",
    "    if mean >= 1000:\n",
    "        use_micros = True\n",
    "        t_df['latency'] = t_df['latency'].astype(float) / 1000\n",
    "\n",
    "    fig.add_trace(go.Scatter(x=t_df['timestamp'], y=t_df['latency'], name=tag))\n",
    "    fig.add_trace(go.Scatter(x=t_df['timestamp'], y=t_df['latency'].rolling(rolling_window).mean(), name=tag + ' mean'))\n",
    "#     fig.add_trace(go.Scatter(x=t_df['timestamp'], y=t_df['latency'].rolling(rolling_window).std(), name=tag + ' std'))\n",
    "\n",
    "    fig.update_layout(title='performance ' + tag + ' ' + ('microseconds' if use_micros else 'nanoseconds'), height=750, width=1000, hovermode='x', legend=dict(\n",
    "        yanchor=\"top\",\n",
    "        y=0.99,\n",
    "        xanchor=\"left\",\n",
    "        x=0.01\n",
    "    ))\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f4d309b",
   "metadata": {},
   "outputs": [],
   "source": [
    "HOPS = [\n",
    "    ['T1_OrderServer_TCP_read', 'T2_OrderServer_LFQueue_write'],\n",
    "    ['T2_OrderServer_LFQueue_write', 'T3_MatchingEngine_LFQueue_read'],\n",
    "    ['T3_MatchingEngine_LFQueue_read', 'T4_MatchingEngine_LFQueue_write'], ['T3_MatchingEngine_LFQueue_read', 'T4t_MatchingEngine_LFQueue_write'],\n",
    "    ['T4_MatchingEngine_LFQueue_write', 'T5_MarketDataPublisher_LFQueue_read'], ['T4t_MatchingEngine_LFQueue_write', 'T5t_OrderServer_LFQueue_read'],\n",
    "    ['T5_MarketDataPublisher_LFQueue_read', 'T6_MarketDataPublisher_UDP_write'], ['T5t_OrderServer_LFQueue_read', 'T6t_OrderServer_TCP_write'],\n",
    "    ['T7_MarketDataConsumer_UDP_read', 'T8_MarketDataConsumer_LFQueue_write'], ['T7t_OrderGateway_TCP_read', 'T8t_OrderGateway_LFQueue_write'],\n",
    "    ['T8_MarketDataConsumer_LFQueue_write', 'T9_TradeEngine_LFQueue_read'], ['T8t_OrderGateway_LFQueue_write', 'T9t_TradeEngine_LFQueue_read'],\n",
    "    ['T9_TradeEngine_LFQueue_read', 'T10_TradeEngine_LFQueue_write'], ['T9t_TradeEngine_LFQueue_read', 'T10_TradeEngine_LFQueue_write'],\n",
    "    ['T10_TradeEngine_LFQueue_write', 'T11_OrderGateway_LFQueue_read'],\n",
    "    ['T11_OrderGateway_LFQueue_read', 'T12_OrderGateway_TCP_write'],\n",
    "    # exchange <-> client\n",
    "    ['T12_OrderGateway_TCP_write', 'T1_OrderServer_TCP_read'],\n",
    "    ['T6_MarketDataPublisher_UDP_write', 'T7_MarketDataConsumer_UDP_read'], ['T6t_OrderServer_TCP_write', 'T7t_OrderGateway_TCP_read'],\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fccbd742",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for tags in HOPS:\n",
    "    tag_p, tag_n = tags\n",
    "    print('{} => {}. {} => {}.'.format(tag_p, len(ttt_df[ttt_df['tag'] == tag_p]), tag_n, len(ttt_df[ttt_df['tag'] == tag_n])))\n",
    "\n",
    "    fig = go.Figure()\n",
    "\n",
    "    t_df = ttt_df[(ttt_df['tag'] == tag_n) | (ttt_df['tag'] == tag_p)]\n",
    "    t_df['latency_diff'] = t_df['latency'].diff()\n",
    "    t_df = t_df[t_df['latency_diff'] > 0]\n",
    "    t_df = t_df[t_df.tag == tag_n]\n",
    "\n",
    "    q_hi = t_df['latency_diff'].quantile(0.99)\n",
    "    q_lo = t_df['latency_diff'].quantile(0.01)\n",
    "    t_df = t_df[(t_df['latency_diff'] < q_hi) & (t_df['latency_diff'] > q_lo)]\n",
    "\n",
    "    mean = t_df['latency_diff'].astype(float).mean()\n",
    "    print('{} has {} observations mean {}'.format(tag_n, len(t_df), mean))\n",
    "\n",
    "    rolling_window = max(1, int(len(t_df) / 100))\n",
    "\n",
    "    unit = 'nanoseconds'\n",
    "    if mean >= 1000000:\n",
    "        unit = 'milliseconds'\n",
    "        t_df['latency_diff'] = t_df['latency_diff'].astype(float) / 1000000\n",
    "    elif mean >= 1000:\n",
    "        unit = 'microseconds'\n",
    "        t_df['latency_diff'] = t_df['latency_diff'].astype(float) / 1000\n",
    "\n",
    "    tag_name = tag_p + ' -> ' + tag_n\n",
    "    fig.add_trace(go.Scatter(x=t_df['timestamp'], y=t_df['latency_diff'], name=tag_name))\n",
    "    fig.add_trace(go.Scatter(x=t_df['timestamp'], y=t_df['latency_diff'].rolling(rolling_window).mean(), name=tag_name + ' mean'))\n",
    "\n",
    "    fig.update_layout(title='performance ' + tag_name + ' ' + unit, height=750, width=1000, hovermode='x', legend=dict(\n",
    "        yanchor=\"top\",\n",
    "        y=0.99,\n",
    "        xanchor=\"left\",\n",
    "        x=0.01\n",
    "    ))\n",
    "    fig.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7fea11e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import session_info\n",
    "session_info.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
